﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Text;
using Utilities;

namespace Tokenization
{
    public class Tokenizer
    {
        public string Input { get; set; }
        public string Output { get; set; }
        public List<TokenRules> Rules { get; set; }

        public Tokenizer()
        {
            Rules = new List<TokenRules>();
        }

        public Hashtable getList(String filename)
        {
            Hashtable htb = new Hashtable();
            StreamReader sr = new StreamReader(filename);
            String line = "";
            while ((line = sr.ReadLine()) != null)
            {
                String[] l = line.Split('\t');
                if (!htb.ContainsKey(l[0])) htb.Add(l[0], l[1]);
            }
            sr.Close();
            return htb;
        }
        public String getRuleData(String filename)
        {
            String res = "(";
            StreamReader sr = new StreamReader(filename);
            String line = "";
            while ((line = sr.ReadLine()) != null)
            {
                if (res == "(") res = res + line;
                else res = res + "|" + line;
            }
            res = res + ")";
            sr.Close();
            return res;
        }

        public String ReplacebyRule(Rule rule, String st)
        {
            String[] ph = st.Split('[', ']');
            String res = rule.ReplacePattern(ph[0]);
            for (int i = 1; i < ph.Length; i++)
            {
                if (i % 2 == 0) res = res + rule.ReplacePattern(ph[i]);
                else res = res + "[" + ph[i] + "]";
            }
            return res;
        }
        public String ReplacebyRuleS(Rule rule, String st)
        {
            String res = rule.ReplacePattern(st);
            return res;
        }

        public void createRules(string Path)
        {
            Debugger.log("Loading Data file...");

            string mBEoP1 = "([\"“”«»‘'?!:,;\\*…])";
            string mBEoP2 = "([\\(\\)<>\"“”«»‘'?!\\-\\.:,;\\*…])";
            string mBEoP3 = "([\\(\\)<>\"“”«»‘'?!\\.:,;\\*…])";
            string number = "([0-9]+([\\.,][0-9]+)*)";
            string number1 = "([0-9]+([\\.,][0-9]+)+)";
            string Consonant = getRuleData(Path + "Spell/Phu Am.txt");
            string Vowel = getRuleData(Path + "Spell/Nguyen Am.txt");

            string UNumber = getRuleData(Path + "Number/H_Number.txt");
            string unit = getRuleData(Path + "Unit/Unit.txt");

            string tt_Score = getRuleData(Path + "Score/TT_Score.txt");

            string Upper = getRuleData(Path + "Tieng Viet/Upper.txt");
            string Upper1 = getRuleData(Path + "Tieng Viet/Upper1.txt");
            string Upper2 = getRuleData(Path + "Tieng Viet/Upper2.txt");
            string Lower = getRuleData(Path + "Tieng Viet/Lower.txt");

            string PA_TV_Upper = getRuleData(Path + "Tieng Viet/phu_am_TV_Upper.txt");
            string PA_TV = getRuleData(Path + "Tieng Viet/phu_am_TV.txt");
            string AT_TV = getRuleData(Path + "Tieng Viet/am_tiet_TV.txt");
            string AT_TV_Lower = getRuleData(Path + "Tieng Viet/am_tiet_TV_Lower.txt");
            string AT_TV_Upper = getRuleData(Path + "Tieng Viet/am_tiet_TV_Upper.txt");
            string TV = getRuleData(Path + "Tieng Viet/tv.txt");

            Debugger.log("Creating...");
            TokenRules tr = new TokenRules("Parser_LV_1");

            string patternI = "(?i)(^|\\s)" + mBEoP1 + "+";
            string patternO = "$0";
            string pattern = mBEoP1 + "+";
            tr.AddToken("BEoP1", true, patternI, patternO, pattern);

            patternI = mBEoP1 + "+(\\s|$)";
            patternO = "$0";
            pattern = mBEoP1 + "+";
            tr.AddToken("BEoP2", true, patternI, patternO, pattern);

            patternI = number;
            patternO = "[$0][Num]";
            pattern = number;
            tr.AddToken("Num", true, patternI, patternO, pattern);

            patternI = @"(^|\W)([A-Z]\.)\s+([A-Z]\.)\s+([A-Z]\.)\s+([A-Z]\w+)(\W|$)";
            patternO = "[$1-$2-$3-$4][Key]";
            pattern = @"([A-Z]\.)\s+([A-Z]\.)\s+([A-Z]\.)\s+([A-Z]\w+)";
            tr.AddToken("NE1", true, patternI, patternO, pattern);

            patternI = @"(^|\W)([A-Z]\.)\s+([A-Z]\.)\s+([A-Z]\w+)(\W|$)";
            patternO = "[$1-$2-$3][Key]";
            pattern = @"([A-Z]\.)\s+([A-Z]\.)\s+([A-Z]\w+)";
            tr.AddToken("NE2", true, patternI, patternO, pattern);

            patternI = @"(^|\W)([A-Z]\.)\s+([A-Z]\w+)(\W|$)";
            patternO = "[$1-$2][Key]";
            pattern = @"([A-Z]\.)\s+([A-Z]\w+)";
            tr.AddToken("NE3", true, patternI, patternO, pattern);

            tr.AddToken("OtherKey", true, @"(?i)\w+(-\w+)+", "[$0][Key]", @"(?i)\w+(-\w+)+");

            patternI = "(?i)((?!((^|\\W)" + TV + "(\\W|$)))(^|\\W)\\w+(\\W|$))";
            patternO = "$1[$2][Key]$3";
            pattern = "(?i)(^|\\W)(\\w+)(\\W|$)";
            tr.AddToken("Key", true, patternI, patternO, pattern);
            tr.AddToken("Key", true, patternI, patternO, pattern);

            patternI = @"([\+\-]+)?\[[^\]]+\]\[(KeyNum|Num)\]\s*[-\./]\s*\[[^\]]+\]\[(KeyNum|Num)\]";
            patternO = "[$1$3$4][KeyNum]";
            pattern = @"\[([^\]]+)\]\[(KeyNum|Num)\](\s*[-\./]\s*)\[([^\]]+)\]\[(KeyNum|Num)\]";
            tr.AddToken("KeyNum", false, patternI, patternO, pattern);
            tr.AddToken("KeyNum", false, patternI, patternO, pattern);
            tr.AddToken("KeyNum", false, patternI, patternO, pattern);

            patternI = "(([^\\s^\\]^\\[^|]*)(\\[[^\\]]+\\])\\[\\w+\\])*([^\\s^\\]^\\[^|]*)" +
            "(\\[[^\\]]+\\])\\[Key\\]([^\\s^\\]^\\[^|]*)((\\[[^\\]]+\\])\\[\\w+\\])*([^\\s^\\]^\\[^|]*)";
            patternO = "[$2$3$5$6$8$9$11$12$14$15$17$18$20$21$22$23$25$26$28$29$31$32$34$35$37$38$40$41$42][Key]";
            pattern = "(([^\\s^\\]^\\[^|]*)\\[([^\\]]+)\\]\\[\\w+\\])?(([^\\s^\\]^\\[^|]*)\\[([^\\]]+)\\]\\[\\w+\\])?" +
                "(([^\\s^\\]^\\[^|]*)\\[([^\\]]+)\\]\\[\\w+\\])?(([^\\s^\\]^\\[^|]*)\\[([^\\]]+)\\]\\[\\w+\\])?" +
                "(([^\\s^\\]^\\[^|]*)\\[([^\\]]+)\\]\\[\\w+\\])?(([^\\s^\\]^\\[^|]*)\\[([^\\]]+)\\]\\[\\w+\\])?" +
                "(([^\\s^\\]^\\[^|]*)\\[([^\\]]+)\\]\\[\\w+\\])?([^\\s^\\]^\\[^|]*)\\[([^\\]]+)\\]\\[Key\\]" +
                "(([^\\s^\\]^\\[^|]*)\\[([^\\]]+)\\]\\[\\w+\\])?(([^\\s^\\]^\\[^|]*)\\[([^\\]]+)\\]\\[\\w+\\])?" +
                "(([^\\s^\\]^\\[^|]*)\\[([^\\]]+)\\]\\[\\w+\\])?(([^\\s^\\]^\\[^|]*)\\[([^\\]]+)\\]\\[\\w+\\])?" +
                "(([^\\s^\\]^\\[^|]*)\\[([^\\]]+)\\]\\[\\w+\\])?(([^\\s^\\]^\\[^|]*)\\[([^\\]]+)\\]\\[\\w+\\])?([^\\s^\\]^\\[]*)";
            tr.AddToken("KeyL1", false, patternI, patternO, pattern);

            patternI = @"(\[[^\]]+\])\[Key\]\s+([A-Z]\w*)\s+(\[[^\]]+\])\[Key\]";
            patternO = "[$1-$2-$3][Key]";
            pattern = @"\[([^\]]+)\]\[Key\]\s+([A-Z]\w*)\s+\[([^\]]+)\]\[Key\]";
            tr.AddToken("KeyL2", false, patternI, patternO, pattern);

            patternI = @"(\[[^\]]+\])\[Key\]\s+([A-Z]\w*)\s+([A-Z]\w*)\s+(\[[^\]]+\])\[Key\]";
            patternO = "[$1-$2-$3-$4][Key]";
            pattern = @"(\[[^\]]+\])\[Key\]\s+([A-Z]\w*)\s+([A-Z]\w*)\s+(\[[^\]]+\])\[Key\]";
            tr.AddToken("KeyL3", false, patternI, patternO, pattern);

            patternI = @"(\[[^\]]+\])\[Key\]\s+([A-Z]\.)\s+(\[[^\]]+\])\[Key\]";
            patternO = "[$1-$2-$3][Key]";
            pattern = @"\[([^\]]+)\]\[Key\]\s+([A-Z]\.)\s+\[([^\]]+)\]\[Key\]";
            tr.AddToken("KeyL4", false, patternI, patternO, pattern);

            patternI = @"(\[[^\]]+\])\[Key\]\s+([A-Z]\.)\s+([A-Z]\.)\s+(\[[^\]]+\])\[Key\]";
            patternO = "[$1-$2-$3-$4][Key]";
            pattern = @"(\[[^\]]+\])\[Key\]\s+([A-Z]\.)\s+([A-Z]\.)\s+(\[[^\]]+\])\[Key\]";
            tr.AddToken("KeyL5", false, patternI, patternO, pattern);

            patternI = @"([^\w^\s^\]^\[^|]*)(\[[^\]]+\])\[(Num|KeyNum)\]([^\w^\s^\]^\[^|]*)";
            patternO = "[$1$2$4][KeyNum]";
            pattern = @"([^\w^\s^\]^\[^|]*)\[([^\]]+)\]\[(Num|KeyNum)\]([^\w^\s^\]^\[^|]*)";
            tr.AddToken("NumL", false, patternI, patternO, pattern);

            patternI = @"\[" + mBEoP2 + @"+\w[^\]]*\]\[Key\]";
            patternO = "$1[$3][Key]";
            pattern = @"\[(" + mBEoP2 + @"+)(\w[^\]]*)\]\[Key\]";
            tr.AddToken("BEoP3", false, patternI, patternO, pattern);

            patternI = @"\[[^\]]+\w" + mBEoP2 + @"+\]\[Key\]";
            patternO = "[$1][Key]$2";
            pattern = @"\[([^\]]+\w)(" + mBEoP2 + @"+)\]\[Key\]";
            tr.AddToken("BEoP4", false, patternI, patternO, pattern);


            TokenRules.serialize(Path + "parser_lv_1.xml",tr);

            tr = new TokenRules("Parser_LV_2");
            patternI = "(\\[[^\\]]+\\])\\[Num\\](\\s+(\\[[^\\]]+\\])\\[Num\\])+";
            patternO = "[$1$2$3$5$6$8$9$11$12$14$15$17$18$20$21$23$24$26$27$29$30$32$33$35$36$38$39$41$42$44$45" +
                "$47$48$50$51$53$54$56$57$59$60$62$63$65$66$68$69][KeyNum]";
            pattern = "\\[([^\\]]+)\\]\\[\\w+\\](\\s+)\\[([^\\]]+)\\]\\[\\w+\\]" +
                "((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?" +
                "((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?" +
                "((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?" +
                "((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?" +
                "((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?" +
                "((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?" +
                "((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?" +
                "((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?" +
                "((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?" +
                "((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?" +
                "((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?((\\s+)\\[([^\\]]+)\\]\\[\\w+\\])?";
            tr.AddToken("Join_KeyNum", false, patternI,patternO,pattern);

            patternI = @"(\[[^\]]+\])\[Key\]\s+(\[[^\]]+\])\[Key\]";
            patternO = "[$1-$2][Key]";
            pattern = @"\[([^\]]+)\]\[Key\]\s+\[([^\]]+)\]\[Key\]";
            tr.AddToken("Join_Key", false, patternI, patternO, pattern);
            tr.AddToken("Join_Key", false, patternI, patternO, pattern);

            patternI = @"(\[[^\]]+\])\[Key\]\s*[-]\s*(\[[^\]]+\])\[Key\]";
            patternO = "[$1-$2][Key]";
            pattern = @"\[([^\]]+)\]\[Key\]\s*[-]\s*\[([^\]]+)\]\[Key\]";
            tr.AddToken("Join_Key1", false, patternI, patternO, pattern);
            tr.AddToken("Join_Key1", false, patternI, patternO, pattern);

            patternI = @"\[" + mBEoP2 + @"+[^\W][^\]]*\]\[(Key|Num|KeyNum)\]";
            patternO = "$1[$3][$4]";
            pattern = @"\[(" + mBEoP2 + @"+)([^\W][^\]]*)\]\[(Key|Num|KeyNum)\]";
            tr.AddToken("Recog_BEoP1", false, patternI, patternO, pattern);

            patternI = @"\[[^\]]*[^\W]" + mBEoP2 + @"+\]\[(Key|Num|KeyNum)\]";
            patternO = "[$1][$4]$2";
            pattern = @"\[([^\]]*[^\W])(" + mBEoP2 + @"+)\]\[(Key|Num|KeyNum)\]";
            tr.AddToken("Recog_BEoP2", false, patternI, patternO, pattern);
            TokenRules.serialize(Path + "parser_lv_2.xml",tr);


            Debugger.log("Ok");
        }
        public TokenRules loadRules(string rulefile) 
        {
            System.Diagnostics.Debug.WriteLine("Loading rules ...");
            TokenRules tr = TokenRules.deserialize(rulefile);
            System.Diagnostics.Debug.WriteLine("Completed. TokenRules name: " + tr.Name);
            Rules.Add(tr);
            return tr;
        }
        public void clearRules()
        {
            Rules.Clear();
        }

        public string Process(string input) {
            Input = input;
            foreach (var tr in Rules)
            {
                foreach (var r in tr.Tokens)
                {
                    if (r.Type)
                        input = ReplacebyRule(r.Rule, input);
                    else
                        input = ReplacebyRuleS(r.Rule, input);
                }
                Output = input;
            }
            return input;
        }
    }
}
